{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "TensorFlow: 静态计算图\n",
    "-------------------------\n",
    "\n",
    "实现一个隐层的全连接神经网络，优化的目标函数是预测值和真实值的欧氏距离。\n",
    "\n",
    "这个实现使用基本的Tensorflow操作来构建一个计算图，然后多次执行这个计算图来训练网络。\n",
    "\n",
    "Tensorflow和PyTorch最大的区别之一就是Tensorflow使用静态计算图和PyTorch使用动态计算图。\n",
    "\n",
    "在Tensorflow里，我们首先构建计算图，然后多次执行它。\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26195068.0\n",
      "20718410.0\n",
      "18857554.0\n",
      "17961840.0\n",
      "16660671.0\n",
      "14397628.0\n",
      "11386147.0\n",
      "8297714.0\n",
      "5686458.5\n",
      "3775949.0\n",
      "2501313.8\n",
      "1694169.4\n",
      "1190028.2\n",
      "872532.56\n",
      "667245.25\n",
      "529367.8\n",
      "432513.94\n",
      "361551.94\n",
      "307399.1\n",
      "264590.22\n",
      "229887.2\n",
      "201180.22\n",
      "176994.42\n",
      "156387.06\n",
      "138673.9\n",
      "123337.19\n",
      "109995.77\n",
      "98336.65\n",
      "88120.695\n",
      "79129.03\n",
      "71187.24\n",
      "64159.363\n",
      "57915.336\n",
      "52356.363\n",
      "47385.73\n",
      "42943.492\n",
      "38966.97\n",
      "35401.312\n",
      "32196.355\n",
      "29318.027\n",
      "26724.338\n",
      "24384.127\n",
      "22272.96\n",
      "20362.863\n",
      "18632.184\n",
      "17064.084\n",
      "15640.318\n",
      "14346.949\n",
      "13170.407\n",
      "12099.08\n",
      "11123.058\n",
      "10232.592\n",
      "9419.991\n",
      "8677.953\n",
      "7999.657\n",
      "7379.1387\n",
      "6810.771\n",
      "6289.5938\n",
      "5811.73\n",
      "5373.2812\n",
      "4970.59\n",
      "4600.3823\n",
      "4260.0186\n",
      "3946.9138\n",
      "3658.5598\n",
      "3393.0151\n",
      "3148.2915\n",
      "2922.6013\n",
      "2714.3457\n",
      "2522.0156\n",
      "2344.3477\n",
      "2180.1072\n",
      "2028.2329\n",
      "1887.7124\n",
      "1757.7014\n",
      "1637.2687\n",
      "1525.637\n",
      "1422.1537\n",
      "1326.1802\n",
      "1237.1\n",
      "1154.3699\n",
      "1077.5269\n",
      "1006.1593\n",
      "939.8011\n",
      "878.1216\n",
      "820.7191\n",
      "767.2896\n",
      "717.5639\n",
      "671.26337\n",
      "628.1088\n",
      "587.9124\n",
      "550.43115\n",
      "515.50366\n",
      "482.92136\n",
      "452.5221\n",
      "424.13965\n",
      "397.6375\n",
      "372.8752\n",
      "349.7413\n",
      "328.11423\n",
      "307.8842\n",
      "288.96826\n",
      "271.27173\n",
      "254.71097\n",
      "239.20464\n",
      "224.69565\n",
      "211.1034\n",
      "198.36859\n",
      "186.43617\n",
      "175.25223\n",
      "164.76839\n",
      "154.9374\n",
      "145.71613\n",
      "137.06961\n",
      "128.9542\n",
      "121.34039\n",
      "114.19378\n",
      "107.48249\n",
      "101.17994\n",
      "95.26107\n",
      "89.703606\n",
      "84.4791\n",
      "79.569954\n",
      "74.95631\n",
      "70.6193\n",
      "66.539764\n",
      "62.703724\n",
      "59.098267\n",
      "55.703827\n",
      "52.511444\n",
      "49.507984\n",
      "46.68108\n",
      "44.02072\n",
      "41.517357\n",
      "39.158573\n",
      "36.938213\n",
      "34.846424\n",
      "32.87691\n",
      "31.022028\n",
      "29.275082\n",
      "27.627861\n",
      "26.076075\n",
      "24.613895\n",
      "23.23513\n",
      "21.935492\n",
      "20.710886\n",
      "19.555939\n",
      "18.466797\n",
      "17.43931\n",
      "16.470858\n",
      "15.557481\n",
      "14.695476\n",
      "13.882082\n",
      "13.1146965\n",
      "12.390834\n",
      "11.707202\n",
      "11.062786\n",
      "10.454311\n",
      "9.879388\n",
      "9.337152\n",
      "8.825156\n",
      "8.341573\n",
      "7.8853006\n",
      "7.4543004\n",
      "7.0473003\n",
      "6.662424\n",
      "6.29942\n",
      "5.956386\n",
      "5.632266\n",
      "5.326095\n",
      "5.0368876\n",
      "4.763451\n",
      "4.505366\n",
      "4.261865\n",
      "4.0341663\n",
      "3.8191009\n",
      "3.61575\n",
      "3.423382\n",
      "3.2412505\n",
      "3.0691173\n",
      "2.9062653\n",
      "2.7524498\n",
      "2.606865\n",
      "2.468917\n",
      "2.3385239\n",
      "2.2151153\n",
      "2.0984473\n",
      "1.987993\n",
      "1.8833864\n",
      "1.7844201\n",
      "1.6907152\n",
      "1.6021547\n",
      "1.5182445\n",
      "1.4386835\n",
      "1.3634965\n",
      "1.2922412\n",
      "1.2248887\n",
      "1.1609156\n",
      "1.1004509\n",
      "1.0431538\n",
      "0.98888195\n",
      "0.93749166\n",
      "0.8887854\n",
      "0.84274405\n",
      "0.7990408\n",
      "0.7576481\n",
      "0.71848834\n",
      "0.6813686\n",
      "0.64622545\n",
      "0.61289287\n",
      "0.5812645\n",
      "0.55128187\n",
      "0.5229714\n",
      "0.49604616\n",
      "0.47050902\n",
      "0.44635788\n",
      "0.42348891\n",
      "0.40175575\n",
      "0.38118765\n",
      "0.36164176\n",
      "0.3431494\n",
      "0.32568067\n",
      "0.30901283\n",
      "0.2932862\n",
      "0.278321\n",
      "0.26409635\n",
      "0.25065306\n",
      "0.23796521\n",
      "0.2258453\n",
      "0.21435916\n",
      "0.20346245\n",
      "0.1931707\n",
      "0.1834231\n",
      "0.1740883\n",
      "0.16532598\n",
      "0.15695663\n",
      "0.14897558\n",
      "0.14151135\n",
      "0.13438192\n",
      "0.12760507\n",
      "0.121158674\n",
      "0.11505054\n",
      "0.109260835\n",
      "0.10376947\n",
      "0.098573\n",
      "0.09362982\n",
      "0.08894624\n",
      "0.08447592\n",
      "0.08024668\n",
      "0.07625224\n",
      "0.07243171\n",
      "0.06883026\n",
      "0.06538303\n",
      "0.06213563\n",
      "0.05903982\n",
      "0.05610995\n",
      "0.05330788\n",
      "0.050651833\n",
      "0.048149124\n",
      "0.045773946\n",
      "0.043490537\n",
      "0.041342087\n",
      "0.039309252\n",
      "0.037365805\n",
      "0.03552231\n",
      "0.033764407\n",
      "0.032095782\n",
      "0.030521758\n",
      "0.029013451\n",
      "0.027594218\n",
      "0.02625289\n",
      "0.0249639\n",
      "0.023746097\n",
      "0.022588702\n",
      "0.021488583\n",
      "0.020439502\n",
      "0.019447679\n",
      "0.018500071\n",
      "0.017608173\n",
      "0.016760869\n",
      "0.015939472\n",
      "0.015165244\n",
      "0.014428581\n",
      "0.013733272\n",
      "0.013070222\n",
      "0.01244783\n",
      "0.011845438\n",
      "0.0112830615\n",
      "0.0107437465\n",
      "0.01023237\n",
      "0.009748549\n",
      "0.009288488\n",
      "0.008853068\n",
      "0.008426533\n",
      "0.008037902\n",
      "0.0076627396\n",
      "0.007304938\n",
      "0.0069624777\n",
      "0.0066408217\n",
      "0.0063312785\n",
      "0.0060388027\n",
      "0.0057672546\n",
      "0.0055059763\n",
      "0.0052511445\n",
      "0.005010678\n",
      "0.0047859713\n",
      "0.0045752767\n",
      "0.0043707658\n",
      "0.0041734097\n",
      "0.003988533\n",
      "0.0038116917\n",
      "0.0036439367\n",
      "0.0034866005\n",
      "0.0033343958\n",
      "0.0031903358\n",
      "0.0030535823\n",
      "0.0029205917\n",
      "0.0027959833\n",
      "0.0026754732\n",
      "0.0025651436\n",
      "0.0024571347\n",
      "0.002354744\n",
      "0.00225644\n",
      "0.0021611762\n",
      "0.0020720917\n",
      "0.001987173\n",
      "0.0019066648\n",
      "0.00183136\n",
      "0.0017563984\n",
      "0.0016848829\n",
      "0.001620123\n",
      "0.0015561725\n",
      "0.0014957578\n",
      "0.001438421\n",
      "0.0013827396\n",
      "0.0013276547\n",
      "0.001278857\n",
      "0.0012315303\n",
      "0.0011845066\n",
      "0.0011411646\n",
      "0.0010974092\n",
      "0.0010559544\n",
      "0.0010202432\n",
      "0.0009824467\n",
      "0.0009477473\n",
      "0.0009149544\n",
      "0.00088238064\n",
      "0.0008502668\n",
      "0.0008213114\n",
      "0.00079298153\n",
      "0.00076618354\n",
      "0.0007400497\n",
      "0.0007147199\n",
      "0.00068989286\n",
      "0.0006681576\n",
      "0.0006455714\n",
      "0.0006243871\n",
      "0.0006043181\n",
      "0.0005843443\n",
      "0.0005659136\n",
      "0.00054774893\n",
      "0.0005311744\n",
      "0.000515028\n",
      "0.0004979828\n",
      "0.00048260976\n",
      "0.00046907313\n",
      "0.0004543942\n",
      "0.000440069\n",
      "0.00042781373\n",
      "0.0004153499\n",
      "0.0004028219\n",
      "0.0003913444\n",
      "0.0003801809\n",
      "0.00036953556\n",
      "0.00035911737\n",
      "0.0003488664\n",
      "0.0003403664\n",
      "0.0003307479\n",
      "0.00032171613\n",
      "0.00031323693\n",
      "0.000305182\n",
      "0.0002967154\n",
      "0.00028956734\n",
      "0.00028174266\n",
      "0.0002747721\n",
      "0.00026742963\n",
      "0.0002602255\n",
      "0.00025421783\n",
      "0.00024768565\n",
      "0.00024133282\n",
      "0.00023573388\n",
      "0.00023044388\n",
      "0.00022473352\n",
      "0.00021908243\n",
      "0.00021347875\n",
      "0.00020891693\n",
      "0.00020366526\n",
      "0.00019943099\n",
      "0.00019488399\n",
      "0.00019025893\n",
      "0.00018575584\n",
      "0.00018150953\n",
      "0.00017784961\n",
      "0.00017363488\n",
      "0.00017009591\n",
      "0.00016647953\n",
      "0.00016252308\n",
      "0.00015877691\n",
      "0.0001554619\n",
      "0.00015214793\n",
      "0.00014877027\n",
      "0.00014589979\n",
      "0.00014312749\n",
      "0.00014016857\n",
      "0.00013734275\n",
      "0.00013445021\n",
      "0.00013191468\n",
      "0.00012965422\n",
      "0.00012717316\n",
      "0.0001247985\n",
      "0.0001225905\n",
      "0.00012004487\n",
      "0.00011772627\n",
      "0.000115543364\n",
      "0.00011350223\n",
      "0.00011134123\n",
      "0.000109292676\n",
      "0.000107091495\n",
      "0.00010527302\n",
      "0.00010305623\n",
      "0.00010099892\n",
      "9.949547e-05\n",
      "9.768303e-05\n",
      "9.610063e-05\n",
      "9.4186566e-05\n",
      "9.280996e-05\n",
      "9.097413e-05\n",
      "8.968272e-05\n",
      "8.7763605e-05\n",
      "8.62943e-05\n",
      "8.479576e-05\n",
      "8.314854e-05\n",
      "8.161491e-05\n",
      "8.037276e-05\n",
      "7.908102e-05\n",
      "7.767421e-05\n",
      "7.654806e-05\n",
      "7.5413576e-05\n",
      "7.410988e-05\n",
      "7.3041265e-05\n",
      "7.189974e-05\n",
      "7.0671194e-05\n",
      "6.965799e-05\n",
      "6.857836e-05\n",
      "6.752396e-05\n",
      "6.655766e-05\n",
      "6.546192e-05\n",
      "6.445975e-05\n",
      "6.356122e-05\n",
      "6.267894e-05\n",
      "6.164607e-05\n",
      "6.0721213e-05\n",
      "5.98013e-05\n",
      "5.9130332e-05\n",
      "5.8310172e-05\n",
      "5.754584e-05\n",
      "5.677087e-05\n",
      "5.5835168e-05\n",
      "5.491927e-05\n",
      "5.4262127e-05\n",
      "5.3368527e-05\n",
      "5.246812e-05\n",
      "5.2072388e-05\n",
      "5.1247753e-05\n",
      "5.0668717e-05\n",
      "4.9832295e-05\n",
      "4.9038983e-05\n",
      "4.8469432e-05\n",
      "4.7892485e-05\n",
      "4.731592e-05\n",
      "4.6592333e-05\n",
      "4.6011013e-05\n",
      "4.533401e-05\n",
      "4.484472e-05\n",
      "4.424288e-05\n",
      "4.3861004e-05\n",
      "4.3249656e-05\n",
      "4.2606025e-05\n",
      "4.2039344e-05\n",
      "4.1705403e-05\n",
      "4.125343e-05\n",
      "4.0653158e-05\n",
      "4.019989e-05\n",
      "3.9631715e-05\n",
      "3.9272607e-05\n",
      "3.8901875e-05\n",
      "3.827111e-05\n",
      "3.786852e-05\n",
      "3.7471138e-05\n",
      "3.6959438e-05\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\n",
    "# 首先构建计算图。\n",
    "\n",
    "# N是batch大小；D_in是输入大小。\n",
    "# H是隐单元个数；D_out是输出大小。\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# 输入和输出是placeholder，在用session执行graph的时候我们会feed进去一个batch的训练数据。\n",
    "x = tf.placeholder(tf.float32, shape=(None, D_in))\n",
    "y = tf.placeholder(tf.float32, shape=(None, D_out))\n",
    "\n",
    "# 创建变量，并且随机初始化。 \n",
    "# 在Tensorflow里，变量的生命周期是整个session，因此适合用它来保存模型的参数。\n",
    "w1 = tf.Variable(tf.random_normal((D_in, H)))\n",
    "w2 = tf.Variable(tf.random_normal((H, D_out)))\n",
    "\n",
    "# Forward pass：计算模型的预测值y_pred \n",
    "# 注意和PyTorch不同，这里不会执行任何计算，而只是”定义“了计算，后面用session.run的时候才会真正的执行计算。\n",
    "h = tf.matmul(x, w1)\n",
    "h_relu = tf.maximum(h, tf.zeros(1))\n",
    "y_pred = tf.matmul(h_relu, w2)\n",
    "\n",
    "# 计算loss \n",
    "loss = tf.reduce_sum((y - y_pred) ** 2.0)\n",
    "\n",
    "# 计算梯度。 \n",
    "grad_w1, grad_w2 = tf.gradients(loss, [w1, w2])\n",
    "\n",
    "# 使用梯度下降来更新参数。assign同样也只是定义更新参数的操作，不会真正的执行。\n",
    "# 在Tensorflow里，更新操作是计算图的一部分；而在PyTorch里，因为是动态的”实时“的计算，\n",
    "# 所以参数的更新只是普通的Tensor计算，不属于计算图的一部分。\n",
    "learning_rate = 1e-6\n",
    "new_w1 = w1.assign(w1 - learning_rate * grad_w1)\n",
    "new_w2 = w2.assign(w2 - learning_rate * grad_w2)\n",
    "\n",
    "# 计算图构建好了之后，我们需要创建一个session来执行计算图。\n",
    "with tf.Session() as sess:\n",
    "    # 首先需要用session初始化变量 \n",
    "    sess.run(tf.global_variables_initializer())\n",
    "\n",
    "    # 这是fake的训练数据\n",
    "    x_value = np.random.randn(N, D_in)\n",
    "    y_value = np.random.randn(N, D_out)\n",
    "    for _ in range(500):\n",
    "        # 用session多次的执行计算图。每次feed进去不同的数据(这里是模拟的，实际应该每次feed一个batch的数据）。\n",
    "        # run的第一个参数是需要执行的计算图的节点，它依赖的节点也会自动执行，因此我们不需要手动执行forward的计算。\n",
    "        # run返回这些节点执行后的值，并且返回的是numpy array\n",
    "        loss_value, _, _ = sess.run([loss, new_w1, new_w2],\n",
    "                                    feed_dict={x: x_value, y: y_value})\n",
    "        print(loss_value)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py3.6-env",
   "language": "python",
   "name": "py3.6-env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
